/******************************************************************************
 *    Copyright 2013 IT2media - IT-Solutions for Directory Business           *
 *                                                                            *
 *  Licensed under the Apache License, Version 2.0 (the "License");           *
 *  you may not use this file except in compliance with the License.          *
 *  You may obtain a copy of the License at                                   *
 *                                                                            *
 *      http://www.apache.org/licenses/LICENSE-2.0                            *
 *                                                                            *
 *  Unless required by applicable law or agreed to in writing, software       *
 *  distributed under the License is distributed on an "AS IS" BASIS,         *
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  *
 *  See the License for the specific language governing permissions and       *
 *  limitations under the License.                                            *
 ******************************************************************************/
package de.it2media.javasitemapgenerator;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;

import org.apache.commons.compress.compressors.CompressorException;
import org.apache.commons.compress.compressors.CompressorOutputStream;
import org.apache.commons.compress.compressors.CompressorStreamFactory;
import org.apache.commons.compress.utils.IOUtils;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import de.it2media.javasitemapgenerator.exceptions.InvalidOutputDirectoryException;
import de.it2media.javasitemapgenerator.header.XMLNS;
import de.it2media.javasitemapgenerator.url.JavaSitemapGeneratorURL;

/**
 * @author Maik Igloffstein <maik.igloffstein@it2media.de>
 * 
 */
public class FileJavaSitemapGenerator extends AbstractJavaSitemapGenerator {

  /**
   * A sex logger for our app
   */
  protected static Logger log                 = LoggerFactory.getLogger(FileJavaSitemapGenerator.class);

  /**
   * Don't write larger files than this maxFileSize
   * UNCOMPRESSED!!!!!
   * 
   * A Sitemap file can contain no more than 50,000 URLs and must be no larger than 50MB when uncompressed. If your Sitemap is larger than this, break it into several smaller Sitemaps. These limits help ensure that your web server is not overloaded by serving large files to Google.
   * 
   * 50 MB
   * 50 * 1024 * 1024 = 52428800 Byte
   * MB KB B
   * 
   */
  public static Integer   MAX_FILE_SIZE       = 50000000;

  /**
   * Limit every file to 10000 url entries
   * 
   * A Sitemap file can contain no more than 50,000 URLs and must be no larger than 50MB when uncompressed. If your Sitemap is larger than this, break it into several smaller Sitemaps. These limits help ensure that your web server is not overloaded by serving large files to Google.
   */
  public static Integer   MAX_URL_PER_FILE    = 50000;

  /**
   * When you've created your Sitemap index file, save it and submit it to Google. As long as you've uploaded and saved all your Sitemaps, you don't need to submit each Sitemap individually. Just submit the Sitemap index file and you're good to go. You can submit up to 500 Sitemap index files for each site in your account.
   */
  public static Integer   MAX_INDEX_FILES     = 500;

  /**
   * A list of all written sitemap files
   */
  protected List<File>    writtenSitemapFiles = new ArrayList<File>();

  /**
   * Zip the output?
   * Default is true
   */
  public boolean          gzip                = true;

  /**
   * The formater for the output
   */
  protected DecimalFormat df                  = new DecimalFormat("00000");

  /**
   * Yes - we will save the files here!
   */
  protected File          outputDirectory;

  /**
   * A nice name for our sitemaps
   */
  protected String        nameForSitemaps     = "sitemap";

  /**
   * The url where google can access our sitemaps later
   */
  protected URL           sitemapsURL;

  /**
   * how many urls did we wrote?
   * because we have to watch the count
   * none at the start
   */
  protected int           urlCountSum         = 0;

  /**
   * We need a output directory<br>
   * and a sitemapFilename, default is sitemap<br>
   * <br>
   * The sitemaps will be wirtten to:<br>
   * for the index file: [outputDirectory]/sitemap_index.xml<br>
   * and for the sitemaps: [outputDirectory]/sitemap_00001.xml<br>
   * <br>
   * 
   * @param outputDirectory
   * @param nameForSitemaps
   * @throws InvalidOutputDirectoryException
   */
  public FileJavaSitemapGenerator(File outputDirectory, URL sitemapURL) throws InvalidOutputDirectoryException {
    this(outputDirectory, sitemapURL, false);
  }

  /**
   * We need a output directory<br>
   * and a sitemapFilename, default is sitemap<br>
   * <br>
   * The sitemaps will be written to:<br>
   * for the index file: [outputDirectory]/sitemap_index.xml<br>
   * and for the sitemaps: [outputDirectory]/sitemap_00001.xml<br>
   * <br>
   * 
   * @param outputDirectory
   * @param nameForSitemaps
   * @param keepSimpleDateFormat
   * @throws InvalidOutputDirectoryException
   */
  public FileJavaSitemapGenerator(File outputDirectory, URL sitemapURL, boolean keepSimpleDateFormat) throws InvalidOutputDirectoryException {

    // for a faster way to create sitemaps
    // we need a big big array :)
    // so we don't have to realloc
    urls = new ArrayList<JavaSitemapGeneratorURL>(MAX_URL_PER_FILE + 1);

    // check/generate the output directory
    checkAndGenerateOutputDirectory(outputDirectory);

    // remember the url
    if (!sitemapURL.toString().endsWith("/")) {
      try {
        sitemapURL = new URL(sitemapURL.toString() + "/");
      } catch (MalformedURLException e) {
        log.error(e.getMessage(), e);
        throw new RuntimeException(e);
      }
    }

    if (!keepSimpleDateFormat) {
      // override the default date format
      simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
    }

    this.sitemapsURL = sitemapURL;
  }

  /**
   * Is GZIP enabled - or not?
   * 
   * @return
   */
  public boolean isGzip() {
    return gzip;
  }

  /**
   * Enables or disables the GZIP compression
   * 
   * @param gzip
   */
  public void setGzip(boolean gzip) {
    this.gzip = gzip;
  }

  /**
   * Returns the current syntax name for the sitemaps
   * 
   * The sitemaps will be wirtten to:<br>
   * for the index file: [outputDirectory]/&lt;NameForSitemaps&gt;_index.xml<br>
   * and for the sitemaps: [outputDirectory]/&lt;NameForSitemaps&gt;_00001.xml<br>
   * <br>
   * 
   * @return
   */
  public String getNameForSitemaps() {
    return nameForSitemaps;
  }

  /**
   * Sets the current syntax name for the sitemaps
   * 
   * The sitemaps will be wirtten to:<br>
   * for the index file: [outputDirectory]/&lt;NameForSitemaps&gt;_index.xml<br>
   * and for the sitemaps: [outputDirectory]/&lt;NameForSitemaps&gt;_00001.xml<br>
   * <br>
   * 
   * @return
   */
  public FileJavaSitemapGenerator setNameForSitemaps(String nameForSitemaps) {
    this.nameForSitemaps = nameForSitemaps;
    return this;
  }

  /**
   * Checks and generates the output directory if needed
   * 
   * @param outputDirectory
   * @throws InvalidOutputDirectoryException
   */
  protected void checkAndGenerateOutputDirectory(File outputDirectory) throws InvalidOutputDirectoryException {

    if (outputDirectory.exists()) {
      if (!outputDirectory.canWrite()) {
        throw new InvalidOutputDirectoryException("Outputdirectory isn't writeable: " + outputDirectory.getAbsolutePath());
      }
    } else {
      if (!outputDirectory.mkdirs()) {
        throw new InvalidOutputDirectoryException("Outputdirectory can't be created: " + outputDirectory.getAbsolutePath());
      }
    }

    this.outputDirectory = outputDirectory;
  }

  /**
   * Just if you like to override the decimal format for the output<br>
   * 
   * @param df
   * @return
   */
  public FileJavaSitemapGenerator setDecimalFormat(DecimalFormat df) {
    this.df = df;
    return this;
  }

  /**
   * Exports the rest of the sitemap and writes and index file
   * 
   * 
   * @param outputDirectory
   * @param filenamePrefix
   * @return
   * @throws IOException
   */
  public List<File> export() throws IOException {

    // write exports
    // as long we are able to export a file, we do it!
    // but this should run only once
    // if we write a sitemap file after 50.000 urls
    while (urls.size() > 0) {
      writeSitemapFile();
    }

    // just for the good ones
    log.info("Wrote " + writtenSitemapFiles.size() + " sitemap files.");

    // gzip sitemap files - if we shall
    writtenSitemapFiles = gzipSitemapFiles();

    log.info("Zipped " + writtenSitemapFiles.size() + " sitemap files.");

    // write sitemap index file
    writtenSitemapFiles = writeSitemapIndexFile();

    log.info("Files overall: " + writtenSitemapFiles.size() + "");

    // return the sitemaps
    return writtenSitemapFiles;
  }

  /**
   * Compresses the list of sitemaps
   * and deletes the uncompressed files
   * 
   * @return
   * @throws IOException
   */
  protected List<File> gzipSitemapFiles() throws IOException {

    // okay, gzip, if we should
    if (gzip) {
      try {

        for (int idx = 0; idx < writtenSitemapFiles.size(); idx++) {

          // we need a output file
          // we just take the input and add .gz
          File output = new File(writtenSitemapFiles.get(idx).getAbsolutePath() + ".gz");

          // load the compression
          CompressorOutputStream createCompressorOutputStream = new CompressorStreamFactory().createCompressorOutputStream(CompressorStreamFactory.GZIP, new FileOutputStream(output));
          // and compress
          IOUtils.copy(new FileInputStream(writtenSitemapFiles.get(idx)), createCompressorOutputStream);

          // close the output stream
          createCompressorOutputStream.close();

          // delete uncompressed file
          writtenSitemapFiles.get(idx).delete();

          // change the context
          writtenSitemapFiles.set(idx, output);
        }
      } catch (CompressorException e) {
        log.error(e.getMessage(), e);
        e.printStackTrace();
      }
    }

    // return the changed or the unchanged writtenSitemapFiles
    return writtenSitemapFiles;
  }

  protected void writeSitemapFile() throws IOException {

    // how many urls are in our sitemap
    int urlCount = 0;

    // do we have to export something?
    if (urls.size() > 0) {

      // this cool file writer writes our sitemaps
      // generate a new file
      // just for the start
      BufferedWriter writer = generateNewFile();

      // how many bytes did we wrote already?
      // because we have to watch the size
      // none at the dawn
      int writtenBytes = (int) FileUtils.sizeOf(writtenSitemapFiles.get(writtenSitemapFiles.size() - 1));

      // now, we have to write
      // we go over the whole urls
      while (urls.size() > 0) {

        // the current sitemap url
        JavaSitemapGeneratorURL url = urls.get(0);

        // get the string from the url
        String urlContent = url.toXMLFragment();

        // this is NOT the perfect way for guessing the file size from a string
        // because java is using unicode and we realy don't know
        // how large the file will be on the file system
        // i think with ascii this will work for utf-8
        // and the unit test will pass
        // so we think that the calculation works and you can use other encodings
        // and the file won't be to large for google
        // remember: Google likes UTF-8 :)
        int addThisCountOfBytesToFile = urlContent.getBytes(fileEncoding).length;

        // and check
        if ((writtenBytes + addThisCountOfBytesToFile) >= FileJavaSitemapGenerator.MAX_FILE_SIZE || urlCount >= FileJavaSitemapGenerator.MAX_URL_PER_FILE) {

          // okay, we wrote a whole sitemap file
          break;
        }

        // sooo, on this point we should have
        // a valid writer
        // so we write something to disk
        writer.write(urlContent);

        // remove the item
        urls.remove(0);

        // count and count
        urlCountSum++;                              // overall
        urlCount++;                                 // current sitemap file
        writtenBytes += addThisCountOfBytesToFile;  // writen bytes in file
      }

      // close the sitemap
      // the sitemap is automaticly opend in generateNewFile()
      closeWriter(writer);
    }
  }

  /**
   * Generates a new file where we can write a sitemap
   * The important header is included!
   * 
   * @throws IOException
   */
  protected BufferedWriter generateNewFile() throws IOException {

    // our new file
    // some content will be in here soon
    File file = new File(this.outputDirectory.getAbsolutePath() + File.separatorChar + this.nameForSitemaps + "_" + df.format(writtenSitemapFiles.size() + 1) + ".xml");

    // log what we will fill
    log.debug("Creating new file: " + file.getAbsolutePath());

    // add the file to the list
    writtenSitemapFiles.add(file);

    // now we create a writer
    // no buffered writer
    // we hope out disk have a writer
    BufferedWriter writer = new BufferedWriter(new FileWriter(file));

    // write the header information
    writer.write(getXMLHeader());

    // sooo, don't forget the urlset stuff with the xmlns
    // add the xmlns entries
    StringBuilder sb = new StringBuilder("<urlset");
    for (Iterator<XMLNS> it = xmlns.iterator(); it.hasNext();) {
      sb.append(" " + it.next().toString());
    }
    sb.append(">\n");
    writer.write(sb.toString());

    // write the bytes so we can count it!
    writer.flush();

    // return the writer
    return writer;
  }

  /**
   * Closes the writer<br>
   * writes &lt;/urlset&gt; finally<br>
   * 
   * @param writer
   * @param b
   * @throws IOException
   */
  protected void closeWriter(BufferedWriter writer) throws IOException {
    // so we have to close this one
    writer.write("</urlset>");

    // close it and out
    writer.close();
  }

  /**
   * Writes the sitemap index file<br>
   * <br>
   * Looks like:<br>
   * &lt;?xml version="1.0" encoding="UTF-8"?&gt;<br>
   * &lt;sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"&gt;<br>
   * __&lt;sitemap&gt;<br>
   * ____&lt;loc&gt;http://www.example.com/sitemap1.xml.gz&lt;/loc&gt;<br>
   * ____&lt;lastmod&gt;2005-01-01&lt;/lastmod&gt;<br>
   * __&lt;/sitemap&gt;<br>
   * __&lt;sitemap&gt;<br>
   * ____&lt;loc&gt;http://www.example.com/sitemap2.xml.gz&lt;/loc&gt;<br>
   * ____&lt;lastmod&gt;2005-01-01&lt;/lastmod&gt;<br>
   * __&lt;/sitemap&gt;<br>
   * &lt;/sitemapindex&gt;<br>
   * 
   * @throws IOException
   * 
   */
  protected List<File> writeSitemapIndexFile() throws IOException {

    // our new file
    // some content will be in here soon
    File file = new File(this.outputDirectory.getAbsolutePath() + File.separatorChar + this.nameForSitemaps + "_index.xml");

    // now we create a writer
    // no buffered writer
    // we hope out disk have a writer
    FileWriter writer = new FileWriter(file);

    // write the header information
    writer.write(getXMLHeader());

    // sooo, don't forget the urlset stuff with the xmlns
    // add the xmlns entries
    writer.write("<sitemapindex xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/siteindex.xsd\" xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n");

    SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");

    // add the files
    for (File sitemapFile : writtenSitemapFiles) {
      writer.write("  <sitemap>\n    <loc>" + this.sitemapsURL.toString() + sitemapFile.getName() + "</loc>\n    <lastmod>" + simpleDateFormat.format(new Date()) + "</lastmod>\n  </sitemap>\n");
    }

    writer.write("</sitemapindex>");
    // close the writer
    writer.close();

    // add index file
    writtenSitemapFiles.add(file);

    // return the files
    // including the index sitemap file
    return writtenSitemapFiles;
  }

  public de.it2media.javasitemapgenerator.JavaSitemapGenerator add(JavaSitemapGeneratorURL url) {

    // add the url to the sitemap generator
    super.add(url);

    // if we could write a file, do it!
    if (currentSize() >= FileJavaSitemapGenerator.MAX_URL_PER_FILE) {
      try {
        writeSitemapFile();
      } catch (IOException e) {
        e.printStackTrace();
        throw new RuntimeException(e);
      }
    }

    return this;
  }

  /**
   * Returns the number of loaded urls
   * including the written ones
   * 
   * @return
   */
  public int size() {
    return this.urlCountSum + super.size();
  }

  /**
   * Returns the number of items currently in the sitemapgenerator
   * (without the written ones)
   * 
   * @return
   */
  public int currentSize() {
    return super.size();
  }
}
